import json
import glob
import os
# Define the input pattern and output file
current_dir = os.path.dirname(os.path.abspath(__file__))
input_dir = os.path.join(current_dir, 'data')

input_pattern = os.path.join(input_dir, "*_results.jsonl")
output_file = os.path.join(input_dir, "sard_all.jsonl")

# Collect all matching files
input_files = glob.glob(input_pattern)

# 初始化统计变量
total_count = 0

# 读取并合并所有文件内容
unique_inputs = set()
merged_data = []

for file in input_files:
    with open(file, 'r', encoding='utf-8') as f:
        for line in f:
            total_count += 1  # 统计总行数
            try:
                data = json.loads(line)
                input_value = data.get('input', None)
                if input_value:
                    # 对 input 进行预处理
                    processed_input = input_value.replace(" ", "").replace("\\n", "").replace("\\t", "").replace("\n", "").replace("\t", "")
                    if processed_input not in unique_inputs:
                        unique_inputs.add(processed_input)
                        merged_data.append(data)
            except json.JSONDecodeError:
                continue

# 将去重后的数据写入输出文件
with open(output_file, 'w', encoding='utf-8') as f:
    for item in merged_data:
        f.write(json.dumps(item, ensure_ascii=False) + '\n')

# 打印统计信息
print(f"Merged {len(input_files)} files into {output_file}.")
print(f"去重前总行数: {total_count}")
print(f"去重后总行数: {len(merged_data)}")